'''
Created on 2011-12-29

@author: ajwang
'''
import sgmllib
from Queue import Queue
from threading import Condition, Thread
from urlparse import urlparse
import socket
socket.setdefaulttimeout(5.0)
from collections import deque
import time
import datetime

from const import CONST

class UrlContainer(object):
    def __init__(self):
        self.urls = {}
        self.files = {}
    
        
    def get_domain_path(self, _url):
        _up = urlparse(_url)
        
        _path = _up.path
        if _up.query != '':
            _path  = _path + '?' + _up.query
        
        if _path == '/':
            _path = ''
            
        return [_up.netloc, _path]
    
    
    def add(self, _base_url, _url):
        _d, _p = self.get_domain_path(_url)
        _d_l = self.urls.get(_d)
        
        if not _d_l:
            self.urls[_d] = {}
            self.files[_d] = open(CONST.DIR + _d, 'w', 0)
            _d_l = self.urls.get(_d)
            
        if not _d_l.get(_p):
            _d_l[_p] = 1
            self.files[_d].write(_d + _p + ', ' + _base_url + '\n')
    
    